import logging
import os
import threading
import uuid
import requests
from core.utils import mongo
from lxml import html
from concurrent.futures import ThreadPoolExecutor

logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')

logger = logging.getLogger()

headers = {
    'user_agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 SLBrowser/9.0.5.12181 SLBChan/11 SLBVPV/64-bit'
}

lock = threading.Lock()  # 创建线程锁

mongo_db = os.getenv("MONGO_DB")
mongo_list = os.getenv("MONGO_LIST")
url = os.getenv("URL")

def get_list(url, fw, db):
    # 获取元素
    response = requests.get(url, headers=headers)
    response.encoding = 'utf-8'
    html_source = html.fromstring(response.text)

    # 获取数据
    li = html_source.xpath('//li[contains(@class,"watch-tile")]/ul/li')[0]
    title = li.xpath('//div[@class="watch-link"]/a/@title')
    link = li.xpath('//div[@class="watch-link"]/a/@href')
    price = li.xpath('//p[@class="money"]/text()')
    try:
        be_fond_of = li.xpath('//div[@class="praise"]//span/em/text()')
    except Exception as e:
        pass
    try:
        review = 0
        review = li.xpath('//div[@class="praise"]//a/text()')
    except Exception as e:
        pass

    # 在写入文件时加锁，确保线程安全
    with lock:
        for t, l, p, f, r in zip(title, link, price, be_fond_of, review):
            uuid1 = uuid.uuid1()
            p = p.replace("¥", "")
            fw.write(f"{uuid1},{t},{l},{p},{f},{r}\n")
            mongo.add_one(db, mongo_list, {'uuid1': str(uuid1), 't': str(t), 'l': str(l), 'p': str(p), 'f': str(f), 'r': str(r)})
            print(uuid1, t, l, p, f, r)


if __name__ == '__main__':

    db, conn = mongo.get_db(mongo_db)
    fw = open("../data/list.csv", mode="a", encoding="utf-8")

    # 使用 ThreadPoolExecutor 来限制最大线程数为 100
    with ThreadPoolExecutor(max_workers=100) as executor:
        for i in range(1, 2393):
            # 提交任务到线程池
            # print(i)
            executor.submit(get_list, f"{url}p{i}.html", fw, db)

    fw.close()
    conn.close()
